{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import bert\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "import itertools\n",
    "import collections\n",
    "import re\n",
    "import random\n",
    "import sentencepiece as spm\n",
    "from unidecode import unidecode\n",
    "from sklearn.utils import shuffle\n",
    "from prepro_utils import preprocess_text, encode_ids, encode_pieces\n",
    "from malaya.text.function import transformer_textcleaning as cleaning\n",
    "from tensorflow.python.estimator.run_config import RunConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/home/husein/alxlnet/topics.json') as fopen:\n",
    "    topics = set(json.load(fopen).keys())\n",
    "\n",
    "list_topics = list(topics)\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('sp10m.cased.bert.model')\n",
    "\n",
    "with open('sp10m.cased.bert.vocab') as fopen:\n",
    "    v = fopen.read().split('\\n')[:-1]\n",
    "v = [i.split('\\t') for i in v]\n",
    "v = {i[0]: i[1] for i in v}\n",
    "\n",
    "\n",
    "class Tokenizer:\n",
    "    def __init__(self, v):\n",
    "        self.vocab = v\n",
    "        pass\n",
    "\n",
    "    def tokenize(self, string):\n",
    "        return encode_pieces(\n",
    "            sp_model, string, return_unicode = False, sample = False\n",
    "        )\n",
    "\n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return [sp_model.PieceToId(piece) for piece in tokens]\n",
    "\n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return [sp_model.IdToPiece(i) for i in ids]\n",
    "\n",
    "\n",
    "tokenizer = Tokenizer(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def F(text):\n",
    "    tokens_a = tokenizer.tokenize(text)\n",
    "    tokens = ['[CLS]'] + tokens_a + ['[SEP]']\n",
    "    input_id = tokenizer.convert_tokens_to_ids(tokens)\n",
    "    input_mask = [1] * len(input_id)\n",
    "    return input_id, input_mask\n",
    "\n",
    "\n",
    "def XY(data):\n",
    "\n",
    "    if len(set(data[1]) & topics) and random.random() > 0.2:\n",
    "        t = random.choice(data[1])\n",
    "        label = 1\n",
    "    else:\n",
    "        s = set(data[1]) | set()\n",
    "        t = random.choice(list(topics - s))\n",
    "        label = 0\n",
    "    X = F(cleaning(data[0]))\n",
    "    Y = F(t)\n",
    "\n",
    "    return X, Y, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/home/husein/alxlnet/testset-keyphrase.json') as fopen:\n",
    "    data = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_initializer(initializer_range = 0.02):\n",
    "    return tf.truncated_normal_initializer(stddev = initializer_range)\n",
    "\n",
    "\n",
    "def get_assignment_map_from_checkpoint(tvars, init_checkpoint):\n",
    "    \"\"\"Compute the union of the current variables and checkpoint variables.\"\"\"\n",
    "    assignment_map = {}\n",
    "    initialized_variable_names = {}\n",
    "\n",
    "    name_to_variable = collections.OrderedDict()\n",
    "    for var in tvars:\n",
    "        name = var.name\n",
    "        m = re.match('^(.*):\\\\d+$', name)\n",
    "        if m is not None:\n",
    "            name = m.group(1)\n",
    "        name_to_variable[name] = var\n",
    "\n",
    "    init_vars = tf.train.list_variables(init_checkpoint)\n",
    "\n",
    "    assignment_map = collections.OrderedDict()\n",
    "    for x in init_vars:\n",
    "        (name, var) = (x[0], x[1])\n",
    "        if 'bert/' + name not in name_to_variable:\n",
    "            continue\n",
    "        assignment_map[name] = name_to_variable['bert/' + name]\n",
    "        initialized_variable_names[name] = 1\n",
    "        initialized_variable_names[name + ':0'] = 1\n",
    "\n",
    "    return (assignment_map, initialized_variable_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 60\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = 1000000\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)\n",
    "learning_rate = 2e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config = modeling.BertConfig.from_json_file(\n",
    "    'bert-base-2020-03-19/bert_config.json'\n",
    ")\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dimension_output = 2,\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.input_masks = tf.placeholder(tf.float32, [None, None])\n",
    "        \n",
    "        self.X_b = tf.placeholder(tf.int32, [None, None])\n",
    "        self.input_masks_b = tf.placeholder(tf.float32, [None, None])\n",
    "        \n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        \n",
    "        with tf.compat.v1.variable_scope('bert', reuse = False):\n",
    "            model = modeling.BertModel(\n",
    "                config = bert_config,\n",
    "                is_training = True,\n",
    "                input_ids = self.X,\n",
    "                input_mask = self.input_masks,\n",
    "                use_one_hot_embeddings = False,\n",
    "            )\n",
    "\n",
    "            summary = model.get_pooled_output()\n",
    "            summary = tf.identity(summary, name = 'summary')\n",
    "            self.summary = summary\n",
    "            \n",
    "        with tf.compat.v1.variable_scope('bert', reuse = True):\n",
    "            model = modeling.BertModel(\n",
    "                config = bert_config,\n",
    "                is_training = True,\n",
    "                input_ids = self.X_b,\n",
    "                input_mask = self.input_masks_b,\n",
    "                use_one_hot_embeddings = False,\n",
    "            )\n",
    "\n",
    "            summary_b = model.get_pooled_output()\n",
    "        \n",
    "        vectors_concat = [summary, summary_b, tf.abs(summary - summary_b)]\n",
    "        vectors_concat = tf.concat(vectors_concat, axis = 1)\n",
    "        \n",
    "        self.logits = tf.layers.dense(vectors_concat, dimension_output)\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        self.cost = tf.reduce_mean(\n",
    "            tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                logits = self.logits, labels = self.Y\n",
    "            )\n",
    "        )\n",
    "        \n",
    "        correct_pred = tf.equal(\n",
    "            tf.argmax(self.logits, 1, output_type = tf.int32), self.Y\n",
    "        )\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dimension_output = 2\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-keyphrase/model.ckpt-170000\n"
     ]
    }
   ],
   "source": [
    "checkpoint = 'bert-base-keyphrase/model.ckpt-170000'\n",
    "saver = tf.train.Saver(var_list = tf.trainable_variables())\n",
    "saver.restore(sess, checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.2224047,  2.9161866]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = F('Kementerian Pertanian dan Industri Makanan menggalakkan pemain industri pertanian menceburi tanaman penting bagi mengurangkan kebergantungan bahan import dari luar negara')\n",
    "\n",
    "o1 = sess.run(\n",
    "    model.summary,\n",
    "    feed_dict = {\n",
    "        model.X: [X[0]],\n",
    "        model.input_masks: [X[1]],\n",
    "    },\n",
    ")\n",
    "\n",
    "Y = F('tanaman jagung')\n",
    "\n",
    "o2 = sess.run(\n",
    "    model.summary,\n",
    "    feed_dict = {\n",
    "        model.X: [Y[0]],\n",
    "        model.input_masks: [Y[1]],\n",
    "    },\n",
    ")\n",
    "\n",
    "sess.run(\n",
    "    model.logits,\n",
    "    feed_dict = {\n",
    "        model.X: [X[0]],\n",
    "        model.input_masks: [X[1]],\n",
    "        model.X_b: [Y[0]],\n",
    "        model.input_masks_b: [Y[1]],\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.61568844]], dtype=float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity(o1, o2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 625/625 [01:14<00:00,  8.40it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "batch_size = 32\n",
    "real_Y,predict_Y = [], []\n",
    "\n",
    "for i in tqdm(range(0, len(data), batch_size)):\n",
    "    batch = data[i: i + batch_size]\n",
    "    X, mask, X_b, mask_b = [], [], [], []\n",
    "    for k in range(len(batch)):\n",
    "        x = F(batch[k][0])\n",
    "        y = F(batch[k][1])\n",
    "        X.append(x[0])\n",
    "        mask.append(x[1])\n",
    "        X_b.append(y[0])\n",
    "        mask_b.append(y[1])\n",
    "        \n",
    "    X = pad_sequences(X, padding = 'post')\n",
    "    mask = pad_sequences(mask, padding = 'post')\n",
    "    X_b = pad_sequences(X_b, padding = 'post')\n",
    "    mask_b = pad_sequences(mask_b, padding = 'post')\n",
    "    \n",
    "    batch_y = [b[2] for b in batch]\n",
    "    \n",
    "    predict_Y += np.argmax(sess.run(model.logits,\n",
    "            feed_dict = {\n",
    "                model.X: X,\n",
    "                model.input_masks: mask,\n",
    "                model.X_b: X_b,\n",
    "                model.input_masks_b: mask_b,\n",
    "            },\n",
    "    ), 1, ).tolist()\n",
    "    \n",
    "    real_Y += batch_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      " not similar    0.99828   0.99670   0.99749     15133\n",
      "     similar    0.98978   0.99466   0.99221      4867\n",
      "\n",
      "    accuracy                        0.99620     20000\n",
      "   macro avg    0.99403   0.99568   0.99485     20000\n",
      "weighted avg    0.99621   0.99620   0.99620     20000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "print(\n",
    "    metrics.classification_report(\n",
    "        real_Y, predict_Y, target_names = ['not similar', 'similar'],\n",
    "        digits = 5\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
